/*
 * Copyright (C) 2017 Hisilicon Limited.
 *
 * This program is free software; you can redistribute it and /or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version
 */
#ifdef CONFIG_GENERIC_BUG
#undef CONFIG_GENERIC_BUG
#endif
#ifdef CONFIG_BUG
#undef CONFIG_BUG
#endif

#ifdef CONFIG_DEBUG_BUGVERBOSE
#undef CONFIG_DEBUG_BUGVERBOSE
#endif

#include <asm/errno.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/pci.h>
#include <linux/delay.h>
#include <linux/version.h>
#include "devdrv_ctrl.h"
#include "devdrv_dma.h"
#include "devdrv_pci.h"
#include "devdrv_msg.h"
#include "devdrv_interface.h"

struct devdrv_ctrl ctrls[MAX_DEV_CNT];
struct devdrv_dev_state_ctrl g_state_ctrl;
struct devdrv_client *clients[DEVDRV_CLIENT_TYPE_MAX] = { NULL, };
struct devdrv_black_callback black_box = {
    .callback = NULL
};
struct devdrv_client_instance clients_instance[DEVDRV_CLIENT_TYPE_MAX][MAX_DEV_CNT];

struct mutex devdrv_ctrl_mutex; /* Used to walk the hash */
struct mutex g_devdrv_remove_rescan_mutex;
u64 g_devdrv_ctrl_hot_reset_status = 0;

#ifndef readq
static inline u64 readq(void __iomem *addr)
{
    return readl(addr) + ((u64)readl(addr + 4) << 32);
}
#endif

#ifndef writeq
static inline void writeq(u64 value, volatile void *addr)
{
    *(volatile u64 *)addr = value;
}
#endif

static void devdrv_pci_stop_and_remove_bus_device_locked(struct pci_dev *dev)
{
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0)
    pci_stop_and_remove_bus_device_locked(dev);
#else
    mutex_lock(&g_devdrv_remove_rescan_mutex);
    pci_stop_and_remove_bus_device(dev);
    mutex_unlock(&g_devdrv_remove_rescan_mutex);
#endif
}

static u32 devdrv_pci_rescan_bus_locked(struct pci_bus *bus)
{
    u32 ret;
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0)
    pci_lock_rescan_remove();
    ret = pci_rescan_bus(bus);
    pci_unlock_rescan_remove();
#else
    mutex_lock(&g_devdrv_remove_rescan_mutex);
    ret = pci_rescan_bus(bus);
    mutex_unlock(&g_devdrv_remove_rescan_mutex);
#endif
    return ret;
}

void devdrv_dev_startup_ctrl_init(void)
{
    int i;
    g_state_ctrl.dev_num = 0;
    g_state_ctrl.first_flag = 0;
    g_state_ctrl.startup_callback = NULL;
    g_state_ctrl.state_notifier_callback = NULL;
    for (i = 0; i < MAX_DEV_CNT; i++) {
        g_state_ctrl.devid[i] = MAX_DEV_CNT;
        g_state_ctrl.hotreset_devid[i] = MAX_DEV_CNT;
    }
}

int devdrv_ctrl_init(void)
{
    u32 i, j;

    if ((memset_s(ctrls, sizeof(ctrls), 0, sizeof(ctrls)) != 0) ||
        (memset_s(clients, sizeof(clients), 0, sizeof(clients)) != 0) ||
        (memset_s(clients_instance, sizeof(clients_instance), 0, sizeof(clients_instance)) != 0)) {
        devdrv_err("memset_s failed\n");
        return -1;
    }

    for (i = 0; i < DEVDRV_CLIENT_TYPE_MAX; i++) {
        for (j = 0; j < MAX_DEV_CNT; j++) {
            mutex_init(&clients_instance[i][j].flag_mutex);
        }
    }
    mutex_init(&devdrv_ctrl_mutex);
    mutex_init(&g_devdrv_remove_rescan_mutex);
    devdrv_dev_startup_ctrl_init();

    return 0;
}

void devdrv_set_ctrl_priv(u32 dev_id, void *priv)
{
    ctrls[dev_id].priv = priv;
}

int devdrv_get_dev_chip_type(u32 dev_id, u32 *chip_type)
{
    *chip_type = HISI_MINI_V1;
    return 0;
}
EXPORT_SYMBOL(devdrv_get_dev_chip_type);

/* cloud ai server, has 2 node(4p system), host side number 0-3 --- node 0, 4-7 --- node 1
   cloud evb, has 2 node(2p system), host side number 0-1 --- node 0, 2-3 --- node 1 */
int devdrv_get_cloud_server_devid(struct devdrv_shr_para __iomem *para)
{
    int dev_id = -1;

    para->dev_num = BOARD_CLOUD_AI_SERVER_DEV_NUM;
    if (para->dev_id >= para->dev_num || para->dev_num == 0) {
        devdrv_err("dev_id %d is out of range %d\n", para->dev_id, para->dev_num);
        return dev_id;
    }

    dev_id = para->dev_num * para->node_id + para->dev_id;

    devdrv_info("device board type %d node_id %d dev id %d use host dev id %d.\n", para->board_type, para->node_id,
        para->dev_id, dev_id);

    if (ctrls[dev_id].priv != NULL) {
        devdrv_err("dev_id %d is already registered \n", dev_id);
        dev_id = -1;
    } else
        ctrls[dev_id].startup_flg = DEVDRV_DEV_STARTUP_PROBED;

    return dev_id;
}
void devdrv_set_devctrl_startup_flag(u32 dev_id, enum devdrv_dev_startup_flag_type flag)
{
    if (dev_id >= MAX_DEV_CNT) {
        devdrv_err("Invalid dev_id:%d\n", dev_id);
        return;
    }
    ctrls[dev_id].startup_flg = flag;
}

STATIC int devdrv_alloc_devid(struct devdrv_ctrl *ctrl)
{
    struct devdrv_shr_para __iomem *para = NULL;
    struct devdrv_pci_ctrl *pci_ctrl = NULL;
    int dev_id = -1;
    int board_type;
    int i;

    pci_ctrl = (struct devdrv_pci_ctrl *)ctrl->priv;

    mutex_lock(&devdrv_ctrl_mutex);

    for (i = 0; i < MAX_DEV_CNT; i++) {
        /* bus may be already inited probe at first time */
        if ((ctrls[i].bus != NULL) && (ctrls[i].bus == ctrl->bus) &&
            (ctrls[i].startup_flg == DEVDRV_DEV_STARTUP_UNPROBED)) {
            ctrls[i].startup_flg = DEVDRV_DEV_STARTUP_PROBED;
            dev_id = i;
            break;
        }
    }

    /* probe init at first time */
    if (dev_id == -1) {
        para = pci_ctrl->shr_para;
        board_type = para->board_type;
        {
            for (i = 0; i < MAX_DEV_CNT; i++) {
                if (ctrls[i].startup_flg == DEVDRV_DEV_STARTUP_UNPROBED) {
                    ctrls[i].startup_flg = DEVDRV_DEV_STARTUP_PROBED;
                    dev_id = i;
                    break;
                }
            }
        }
    }

    mutex_unlock(&devdrv_ctrl_mutex);

    return dev_id;
}
/* called after probed */
struct devdrv_ctrl *devdrv_get_top_half_devctrl_by_id(u32 dev_id)
{
    u32 i;

    for (i = 0; i < MAX_DEV_CNT; i++) {
        if (((ctrls[i].startup_flg == DEVDRV_DEV_STARTUP_TOP_HALF_OK) ||
            (ctrls[i].startup_flg == DEVDRV_DEV_STARTUP_BOTTOM_HALF_OK)) &&
            (ctrls[i].dev_id == dev_id)) {
            return &ctrls[i];
        }
    }

    return NULL;
}
/* called after half probed */
struct devdrv_ctrl *devdrv_get_bottom_half_devctrl_by_id(u32 dev_id)
{
    u32 i;

    for (i = 0; i < MAX_DEV_CNT; i++) {
        if ((ctrls[i].startup_flg == DEVDRV_DEV_STARTUP_BOTTOM_HALF_OK) && (ctrls[i].dev_id == dev_id)) {
            return &ctrls[i];
        }
    }

    return NULL;
}

/* device boot status */
int devdrv_get_device_boot_status(u32 devid, u32 *boot_status)
{
    struct devdrv_pci_ctrl *pci_ctrl = NULL;
    struct devdrv_ctrl *ctrl = NULL;

    if (devid >= MAX_DEV_CNT) {
        devdrv_err("dev id is invalid\n");
        return -EINVAL;
    }
    if (boot_status == NULL) {
        devdrv_err("boot_status is null\n");
        return -EINVAL;
    }

    ctrl = devdrv_get_top_half_devctrl_by_id(devid);
    if (ctrl == NULL) {
        devdrv_warn("get devctrl by id err:%u!\n", devid);
        *boot_status = DSMI_BOOT_STATUS_UNINIT;
        return -ENXIO;
    }

    pci_ctrl = ctrl->priv;

    *boot_status = pci_ctrl->device_boot_status;
    return 0;
}
EXPORT_SYMBOL(devdrv_get_device_boot_status);
/* record probed mini */
void drvdrv_dev_startup_record(u32 dev_id)
{
    u32 find_flag = 0;
    u32 i;

    mutex_lock(&devdrv_ctrl_mutex);
    /* not the first time probe(hot reset) */
    for (i = 0; i < g_state_ctrl.dev_num; i++) {
        if (dev_id == g_state_ctrl.devid[i]) {
            find_flag = 1;
            break;
        }
    }
    /* the first time probe */
    if (find_flag == 0) {
        g_state_ctrl.devid[g_state_ctrl.dev_num] = dev_id;
        if (g_state_ctrl.dev_num < MAX_DEV_CNT) {
            g_state_ctrl.dev_num++;
        }
        devdrv_info("probe new dev %u, add to report,dev_num:%u.\n", dev_id, g_state_ctrl.dev_num);
    } else {
        devdrv_info("dev %u no need record, just report. dev_num:%u\n", dev_id, g_state_ctrl.dev_num);
    }

    mutex_unlock(&devdrv_ctrl_mutex);
}

/*
report fomat(eg.8mini):
when first register,probe 5mini:
5,[0,1,2,3,4],5
then:
6,[5],1
7,[6],1
8,[7],1
after hot reset devid=2:
8,[2],1
after hot reset devid=4:
8,[4],1
*/
void drvdrv_dev_startup_report(u32 dev_id)
{
    mutex_lock(&devdrv_ctrl_mutex);
    if (g_state_ctrl.startup_callback != NULL) {
        /* report all probed mini when first time reset */
        if (g_state_ctrl.first_flag == 0) {
            if (g_state_ctrl.dev_num != 0) {
                (g_state_ctrl.startup_callback)(g_state_ctrl.dev_num, g_state_ctrl.devid, MAX_DEV_CNT,
                    g_state_ctrl.dev_num);
            }
            devdrv_info("dev startup report num:%u\n", g_state_ctrl.dev_num);
            g_state_ctrl.first_flag = 1;
        }
        /* report one probed mini when hotreset */
        else {
            g_state_ctrl.hotreset_devid[0] = dev_id;
            (g_state_ctrl.startup_callback)(g_state_ctrl.dev_num, g_state_ctrl.hotreset_devid, MAX_DEV_CNT, 1);
            devdrv_info("dev startup report id:%u\n", dev_id);
        }
    } else {
        devdrv_info("dev startup no report id:%u\n", dev_id);
    }
    mutex_unlock(&devdrv_ctrl_mutex);

    return;
}

void drvdrv_dev_state_notifier(struct devdrv_pci_ctrl *pci_ctrl)
{
    u32 dev_id;

    if (pci_ctrl == NULL) {
        devdrv_info("pci_ctrl ptr is null\n");
        return;
    }

    /* when pcie ko removed, stop notify dev manager, and set callback NULL */
    if (pci_ctrl->module_exit_flag == DEVDRV_REMOVE_CALLED_BY_MODULE_EXIT) {
        devdrv_err("Do not notify dev manager driver remove\n");
        g_state_ctrl.state_notifier_callback = NULL;
        return;
    }

    dev_id = pci_ctrl->dev_id;
    if (dev_id >= MAX_DEV_CNT) {
        devdrv_err("dev_id is invalid, %u\n", dev_id);
        return;
    }

    if (g_state_ctrl.state_notifier_callback != NULL) {
        (g_state_ctrl.state_notifier_callback)(dev_id, (u32)GOING_TO_DISABLE_DEV);
        /* add 1s for bbox to dump when unbind */
        ssleep(1);
    } else {
        devdrv_err("state_notifier_callback is NULL, dev_id:%u\n", dev_id);
    }
}


/* for dev manager to register when insmod,then pcie report dev startup info to dev manager */
void drvdrv_dev_startup_register(devdrv_dev_startup_notify startup_callback)
{
    if (startup_callback == NULL) {
        devdrv_warn("startup callback is NULL\n");
        return;
    }

    devdrv_info("startup register, call startup report\n");
    g_state_ctrl.startup_callback = startup_callback;
    drvdrv_dev_startup_report(MAX_DEV_CNT);
}
EXPORT_SYMBOL(drvdrv_dev_startup_register);

/* for dev manager to register when insmod,then pcie report dev remove or other info to dev manager */
void drvdrv_dev_state_notifier_register(devdrv_dev_state_notify state_callback)
{
    if (state_callback == NULL) {
        devdrv_info("state callback is null\n");
        return;
    }
    devdrv_info("state notifier register\n");
    g_state_ctrl.state_notifier_callback = state_callback;
}
EXPORT_SYMBOL(drvdrv_dev_state_notifier_register);

void devdrv_hb_broken_stop_msg_send(u32 devid)
{
    struct devdrv_ctrl *ctrl = NULL;
    struct devdrv_pci_ctrl *pci_ctrl = NULL;

    if (devid >= MAX_DEV_CNT) {
        devdrv_info("device id invalid:%d\n", devid);
        return;
    }

    ctrl = devdrv_get_bottom_half_devctrl_by_id(devid);
    if (ctrl == NULL) {
        devdrv_info("pci dev devid parameter is error\n");
        return;
    }
    pci_ctrl = (struct devdrv_pci_ctrl *)ctrl->priv;

    devdrv_set_device_status(pci_ctrl, DEVDRV_DEVICE_DEAD);

    devdrv_info("heartbeat_broken stop msg end. dev_id %d\n", devid);
}
EXPORT_SYMBOL(devdrv_hb_broken_stop_msg_send);


void devdrv_set_device_boot_status(struct devdrv_pci_ctrl *pci_ctrl, u32 status)
{
    pci_ctrl->device_boot_status = status;
}

int devdrv_get_pcie_status_by_id(u32 dev_id)
{
    struct devdrv_ctrl *ctrl;

    ctrl = devdrv_get_bottom_half_devctrl_by_id(dev_id);
    if (ctrl == NULL) {
        devdrv_err("pcie status not startup ok, dev_id: %d\n", dev_id);
        return -1;
    }
    return 0;
}

int devdrv_pcie_reinit(u32 dev_id)
{
    struct pci_bus *bus = NULL;
    u32 ret;

    if (dev_id >= MAX_DEV_CNT) {
        devdrv_err("dev id is invalid\n");
        return -EINVAL;
    }
    bus = ctrls[dev_id].bus;

    if (bus == NULL) {
        devdrv_err("bus does not exist\n");
        return -EIO;
    }
    devdrv_info("call pcie_reinit, dev_id: %u\n", dev_id);

    if (bus->self != NULL) {
        bus = bus->self->bus;
    }
    ret = devdrv_pci_rescan_bus_locked(bus);
    if (ret == 0) {
        devdrv_err("rescan bus falied\n");
        return -EIO;
    }

    return 0;
}
EXPORT_SYMBOL(devdrv_pcie_reinit);

struct devdrv_dma_dev *devdrv_get_dma_dev(u32 dev_id)
{
    struct devdrv_ctrl *ctrl = devdrv_get_bottom_half_devctrl_by_id(dev_id);
    struct devdrv_pci_ctrl *pci_ctrl = NULL;

    if (ctrl == NULL) {
        devdrv_err("add_dev: access null pointer!\n");
        return NULL;
    }
    pci_ctrl = ctrl->priv;

    return pci_ctrl->dma_dev;
}

u32 devdrv_get_devid_by_dev(struct devdrv_msg_dev *msg_dev)
{
    struct devdrv_pci_ctrl *pci_ctrl = NULL;
    u32 i;

    if (msg_dev == NULL) {
        devdrv_err("msg_dev is null\n");
        return MAX_DEV_CNT;
    }
    for (i = 0; i < MAX_DEV_CNT; i++) {
        pci_ctrl = (struct devdrv_pci_ctrl *)ctrls[i].priv;
        if (pci_ctrl == NULL) {
            continue;
        }
        if (pci_ctrl->msg_dev == msg_dev) {
            return ctrls[i].dev_id;
        }
    }

    devdrv_err("find devid failed!\n");

    return MAX_DEV_CNT;
}

/* Store the ctrl info, and return an id to register caller */
int devdrv_slave_dev_add(struct devdrv_ctrl *ctrl)
{
    int dev_id;

    dev_id = devdrv_alloc_devid(ctrl);

    if (dev_id < 0 || dev_id >= MAX_DEV_CNT) {
        devdrv_err("dev link already full!dev_id:%d\n", dev_id);
        return -1;
    }

    ctrls[(u32)dev_id].dev_id = (u32)dev_id;
    ctrls[(u32)dev_id].priv = ctrl->priv;
    ctrls[(u32)dev_id].dev_type = ctrl->dev_type;
    ctrls[(u32)dev_id].dev = ctrl->dev;
    ctrls[(u32)dev_id].bus = ctrl->bus;
    ctrls[(u32)dev_id].pdev = ctrl->pdev;

    return dev_id;
}
int devdrv_call_init_instance(struct devdrv_ctrl *ctrl)
{
    struct devdrv_client_instance *instance = NULL;
    u32 dev_id;
    u32 ret;
    u32 i;

    dev_id = ctrl->dev_id;

    for (i = 0; i < DEVDRV_CLIENT_TYPE_MAX && dev_id < MAX_DEV_CNT; i++) {
        if (clients[i] == NULL) {
            continue;
        }

        instance = &clients_instance[i][dev_id];
        instance->dev_ctrl = &ctrls[dev_id];
        if (clients[i]->init_instance) {
            mutex_lock(&instance->flag_mutex);
            if (instance->flag == 0) {
                instance->flag = 1;
                mutex_unlock(&instance->flag_mutex);

                devdrv_debug("slave_dev_add,ctrl:%u,dev_id:%u,before init_instance!\n", i, dev_id);
                ret = clients[i]->init_instance(instance);
                devdrv_debug("slave_dev_add,after init_instance!\n");
                if (ret) {
                    devdrv_info("client %u init fialed!\n", i);
                }
            } else {
                mutex_unlock(&instance->flag_mutex);
            }
        }
    }

    return 0;
}

void devdrv_dev_offline_notify(u32 dev_id)
{
    struct devdrv_ctrl *devdrv_ctrl = NULL;
    struct devdrv_pci_ctrl *ctrl = NULL;

    devdrv_ctrl = devdrv_get_bottom_half_devctrl_by_id(dev_id);
    if (devdrv_ctrl == NULL) {
        devdrv_err("can't find dev %u \n", dev_id);
        return;
    }

    ctrl = (struct devdrv_pci_ctrl *)devdrv_ctrl->priv;
    /* notice device */
    (void)devdrv_notify_dev_online(ctrl->msg_dev, dev_id, DEVDRV_DEV_OFFLINE);
}

void devdrv_slave_dev_delete(u32 dev_id)
{
    struct devdrv_client_instance *instance = NULL;
    struct devdrv_ctrl *dev_ctrl = NULL;
    int ret;
    int dev_index = -1;
    u32 i;

    if (dev_id >= MAX_DEV_CNT) {
        devdrv_err("unregister dev: dev id %d not exist !\n", dev_id);
        return;
    }

    for (i = 0; i < MAX_DEV_CNT; i++)
        if (ctrls[i].dev_id == dev_id) {
            dev_ctrl = &ctrls[i];
            dev_index = i;
            break;
        }

    if (i == MAX_DEV_CNT) {
        devdrv_err(" find devctrl failed. dev_id %d\n", dev_id);
        return;
    }

    for (i = 0; i < DEVDRV_CLIENT_TYPE_MAX; i++) {
        if (clients[i] == NULL) {
            continue;
        }

        instance = &clients_instance[i][dev_index];
        if (clients[i]->uninit_instance) {
            mutex_lock(&instance->flag_mutex);
            instance->flag = 0;
            mutex_unlock(&instance->flag_mutex);
            devdrv_info("dev_id %u, client %u uninit start!\n", dev_id, i);
            ret = clients[i]->uninit_instance(instance);
            devdrv_info("dev_id %u, client %u uninit end!\n", dev_id, i);
            if (ret) {
                devdrv_info("dev_id %d,client %d uninit failed!\n", i, dev_id);
            }
            instance->dev_ctrl = NULL;
        }
    }

    devdrv_dev_offline_notify(dev_id);

    if (dev_ctrl != NULL) {
        dev_ctrl->startup_flg = DEVDRV_DEV_STARTUP_UNPROBED;
        dev_ctrl->priv = NULL;
        dev_ctrl->ops = NULL;
        dev_ctrl->dev = NULL;
    }
}

void *devdrv_pcimsg_alloc_trans_queue(u32 dev_id, struct devdrv_trans_msg_chan_info *chan_info)
{
    struct devdrv_ctrl *ctrl = devdrv_get_bottom_half_devctrl_by_id(dev_id);
    if (ctrl == NULL) {
        devdrv_err("can't find dev %u \n", dev_id);
        return NULL;
    }

    if (chan_info == NULL) {
        devdrv_err("chan_info is null!\n");
        return NULL;
    }

    return (void *)(ctrl->ops->alloc_trans_chan(ctrl->priv, chan_info));
}
EXPORT_SYMBOL(devdrv_pcimsg_alloc_trans_queue);

int devdrv_pcimsg_realease_trans_queue(void *msg_chan)
{
    if (msg_chan == NULL) {
        devdrv_err("msg_chan is null!\n");
        return -EINVAL;
    }

    return devdrv_free_trans_queue((struct devdrv_msg_chan *)msg_chan);
}
EXPORT_SYMBOL(devdrv_pcimsg_realease_trans_queue);

/* non-trans msg chan */
void *devdrv_pcimsg_alloc_non_trans_queue(u32 dev_id, struct devdrv_non_trans_msg_chan_info *chan_info)
{
    struct devdrv_ctrl *ctrl = devdrv_get_bottom_half_devctrl_by_id(dev_id);
    if (ctrl == NULL) {
        devdrv_err("can't find dev %u \n", dev_id);
        return NULL;
    }

    if (chan_info == NULL) {
        devdrv_err("chan_info is null!\n");
        return NULL;
    }

    return (void *)(ctrl->ops->alloc_non_trans_chan(ctrl->priv, chan_info));
}
EXPORT_SYMBOL(devdrv_pcimsg_alloc_non_trans_queue);

int devdrv_pcimsg_free_non_trans_queue(void *msg_chan)
{
    if (msg_chan == NULL) {
        devdrv_err("msg_chan is NULL!\n");
        return -EINVAL;
    }

    return devdrv_free_non_trans_queue((struct devdrv_msg_chan *)msg_chan);
}
EXPORT_SYMBOL(devdrv_pcimsg_free_non_trans_queue);

int devdrv_register_black_callback(struct devdrv_black_callback *black_callback)
{
    if (black_callback == NULL) {
        devdrv_err("black_callback is null.\n");
        return -EINVAL;
    }

    if (black_callback->callback == NULL) {
        devdrv_err("callback is null.\n");
        return -EINVAL;
    }

    black_box.callback = black_callback->callback;

    return 0;
}
EXPORT_SYMBOL(devdrv_register_black_callback);

void devdrv_unregister_black_callback(struct devdrv_black_callback *black_callback)
{
    if (black_callback == NULL) {
        devdrv_err("black_callback is null.\n");
        return;
    }

    if (black_callback->callback != black_box.callback) {
        devdrv_err("callback is not same.\n");
        return;
    }

    black_box.callback = NULL;

    return;
}
EXPORT_SYMBOL(devdrv_unregister_black_callback);

int devdrv_register_client(struct devdrv_client *client)
{
    u32 ret;
    u32 dev_id;
    struct devdrv_client_instance *instance = NULL;

    if (client == NULL) {
        devdrv_err("client is null.\n");
        return -EINVAL;
    }

    if (client->type >= DEVDRV_CLIENT_TYPE_MAX) {
        devdrv_err("client type %d is error.\n", client->type);
        return -EINVAL;
    }

    if (clients[client->type] != NULL) {
        devdrv_err("client type %d is already registered.\n", client->type);
        return -EINVAL;
    }

    for (dev_id = 0; dev_id < MAX_DEV_CNT; dev_id++) {
        if (ctrls[dev_id].startup_flg == DEVDRV_DEV_STARTUP_BOTTOM_HALF_OK) {
            instance = &clients_instance[client->type][dev_id];
            instance->dev_ctrl = &ctrls[dev_id];
            if (client->init_instance == NULL) {
                continue;
            }
            devdrv_info("devdrv_register_client,client type:%d,dev_id:%d,before init_instance!\n", client->type,
                dev_id);
            mutex_lock(&instance->flag_mutex);
            if (instance->flag == 0) {
                instance->flag = 1;
                mutex_unlock(&instance->flag_mutex);
                ret = client->init_instance(instance);
                devdrv_info("devdrv_register_client,after init_instance!\n");
                if (ret) {
                    mutex_lock(&instance->flag_mutex);
                    instance->flag = 0;
                    mutex_unlock(&instance->flag_mutex);
                    devdrv_err("init instance ctrl %u failed,delete client!\n", dev_id);
                    return ret;
                }
            } else {
#ifndef DRV_UT

                mutex_unlock(&instance->flag_mutex);
#endif
            }
        }
    }

    clients[client->type] = client;

    return 0;
}
EXPORT_SYMBOL(devdrv_register_client);


int devdrv_unregister_client(struct devdrv_client *client)
{
    struct devdrv_ctrl *dev_ctrl = NULL;
    struct devdrv_client_instance *instance = NULL;
    int dev_id;

    if (client == NULL) {
        devdrv_err("client is NULL!\n");
        return -EINVAL;
    }

    if (client->type >= DEVDRV_CLIENT_TYPE_MAX) {
        devdrv_err("client type %d is error.\n", client->type);
        return -EINVAL;
    }

    for (dev_id = 0; dev_id < MAX_DEV_CNT; dev_id++) {
        instance = &clients_instance[client->type][dev_id];
        dev_ctrl = instance->dev_ctrl;
        if (dev_ctrl != NULL) {
            if (client->uninit_instance == NULL)
                continue;
            mutex_lock(&instance->flag_mutex);
            instance->flag = 0;
            mutex_unlock(&instance->flag_mutex);
            devdrv_info("dev_id %d, client->type %u uninit_instance start!\n", dev_id, client->type);
            client->uninit_instance(instance);
            devdrv_info("dev_id %d, client->type %u uninit_instance end!\n", dev_id, client->type);
            instance->dev_ctrl = NULL;
            instance->priv = NULL;
        }
    }
    clients[client->type] = NULL;
    return 0;
}
EXPORT_SYMBOL(devdrv_unregister_client);

u64 devdrv_get_bar45_len(struct devdrv_pci_ctrl *pci_ctrl)
{
    u64 len = 0;

    len = pci_resource_len(pci_ctrl->pdev, PCI_BAR_MEM);

    return len;
}

int devdrv_get_ts_sq_offset(struct devdrv_pci_ctrl *pci_ctrl, u64 *offset)
{
    int ret = 0;
    u64 len;

    len = devdrv_get_bar45_len(pci_ctrl);
    switch (len) {
        case MEM_BAR_LEN_64M:
            *offset = DEVDRV_BAR4_RESERVE_TS_SQ_OFFSET_64M;
            break;
        case MEM_BAR_LEN_128M:
            *offset = DEVDRV_BAR4_RESERVE_TS_SQ_OFFSET_128M;
            break;
        case MEM_BAR_LEN_4G:
            *offset = DEVDRV_BAR4_RESERVE_TS_SQ_OFFSET_4G;
            break;
        case MEM_BAR_LEN_8G:
            *offset = DEVDRV_BAR4_RESERVE_TS_SQ_OFFSET_8G;
            break;
        case MEM_BAR_LEN_16G:
            *offset = DEVDRV_BAR4_RESERVE_TS_SQ_OFFSET_16G;
            break;
        default:
            devdrv_err("BAR size is 0x%llx\n", len);
            ret = -EINVAL;
            break;
    }

    return ret;
}

int devdrv_get_test_addr_offset(struct devdrv_pci_ctrl *pci_ctrl, u64 *offset)
{
    int ret = 0;
    u64 len;
    len = devdrv_get_bar45_len(pci_ctrl);
    switch (len) {
        case MEM_BAR_LEN_8M:
            *offset = DEVDRV_BAR4_RESERVE_TEST_OFFSET_8M;
            break;
        case MEM_BAR_LEN_64M:
            *offset = DEVDRV_BAR4_RESERVE_TEST_OFFSET_64M;
            break;
        case MEM_BAR_LEN_128M:
            *offset = DEVDRV_BAR4_RESERVE_TEST_OFFSET_128M;
            break;
        case MEM_BAR_LEN_4G:
            *offset = DEVDRV_BAR4_RESERVE_TEST_OFFSET_4G;
            break;
        case MEM_BAR_LEN_8G:
            *offset = DEVDRV_BAR4_RESERVE_TEST_OFFSET_8G;
            break;
        case MEM_BAR_LEN_16G:
            *offset = DEVDRV_BAR4_RESERVE_TEST_OFFSET_16G;
            break;
        default:
            devdrv_err("BAR size is 0x%llx\n", len);
            ret = -EINVAL;
            break;
    }
    return ret;
}

static int get_cut_bar_addr(struct devdrv_pci_ctrl *pci_ctrl, enum devdrv_addr_type type, u64 *addr, size_t *size)
{
    int ret = 0;
    u64 offset = 0;

    switch (type) {
        case DEVDRV_ADDR_TEST_BASE:
            ret = devdrv_get_test_addr_offset(pci_ctrl, &offset);
            if (ret != 0) {
                devdrv_err("get addr offset error! ret %d.\n", ret);
                return -EINVAL;
            }
            *addr = pci_ctrl->mem_phy_base + offset;
            *size = DEVDRV_BAR4_RESERVE_TEST_SIZE;
            break;
        case DEVDRV_ADDR_LOAD_RAM:
            *addr = pci_ctrl->io_phy_base + CUT_DEVDRV_IO_LOAD_SRAM_OFFSET;
            *size = CUT_DEVDRV_IO_LOAD_SRAM_SIZE;
            break;
        default:
            devdrv_err("type parameter is error, type is %d.\n", type);
            ret = -EINVAL;
            break;
    }
    return ret;
}

static int get_no_cut_bar_addr(struct devdrv_pci_ctrl *pci_ctrl, enum devdrv_addr_type type, u64 *addr, size_t *size)
{
    int ret = 0;
    u64 offset = 0;

    switch (type) {
        case DEVDRV_ADDR_TS_DOORBELL:
            *addr = pci_ctrl->io_phy_base + DEVDRV_IO_TS_DB_OFFSET;
            *size = DEVDRV_IO_TS_DB_SIZE;
            break;
        case DEVDRV_ADDR_TS_SRAM:
            *addr = pci_ctrl->io_phy_base + DEVDRV_IO_TS_SRAM_OFFSET;
            *size = DEVDRV_IO_TS_SRAM_SIZE;
            break;
        case DEVDRV_ADDR_TS_SQ_BASE:
            ret = devdrv_get_ts_sq_offset(pci_ctrl, &offset);
            if (ret != 0) {
                devdrv_err("get addr offset error! ret %d.\n", ret);
                return -EINVAL;
            }
            *addr = pci_ctrl->mem_phy_base + offset;
            *size = DEVDRV_BAR4_RESERVE_TS_SQ_SIZE;
            break;
        case DEVDRV_ADDR_TEST_BASE:
            ret = devdrv_get_test_addr_offset(pci_ctrl, &offset);
            if (ret != 0) {
                devdrv_err("get addr offset error! ret %d.\n", ret);
                return -EINVAL;
            }
            *addr = pci_ctrl->mem_phy_base + offset;
            *size = DEVDRV_BAR4_RESERVE_TEST_SIZE;
            break;
        case DEVDRV_ADDR_LOAD_RAM:
            *addr = pci_ctrl->io_phy_base + DEVDRV_IO_LOAD_SRAM_OFFSET;
            *size = DEVDRV_IO_LOAD_SRAM_SIZE;
            break;
        case DEVDRV_ADDR_BBOX_HDR_BASE:
            *addr = pci_ctrl->io_phy_base + DEVDRV_IO_BBOX_HDR_OFFSET;
            *size = DEVDRV_IO_BBOX_HDR_SIZE;
            break;
        default:
            devdrv_err("type parameter is error.\n");
            ret = -EINVAL;
            break;
    }
    return ret;
}

int devdrv_get_addr_info(u32 devid, enum devdrv_addr_type type, u32 index, u64 *addr, size_t *size)
{
    struct devdrv_ctrl *ctrl = NULL;
    struct devdrv_pci_ctrl *pci_ctrl = NULL;
    int ret = 0;

    if (index || addr == NULL || size == NULL) {
        devdrv_err("parameter is error.\n");
        return -EINVAL;
    }

    ctrl = devdrv_get_top_half_devctrl_by_id(devid);
    if (ctrl == NULL) {
        devdrv_err("devid parameter is error, devid:%d.\n", devid);
        return -EINVAL;
    }

    pci_ctrl = (struct devdrv_pci_ctrl *)ctrl->priv;
    if (pci_ctrl == NULL) {
        devdrv_err("ctrl->priv parameter is error, devid:%d.\n", devid);
        return -EINVAL;
    }

    if (pci_ctrl->device_bar_cut_flag) {
        ret = get_cut_bar_addr(pci_ctrl, type, addr, size);
    } else {
        ret = get_no_cut_bar_addr(pci_ctrl, type, addr, size);
    }

    return ret;
}
EXPORT_SYMBOL(devdrv_get_addr_info);

int get_device_bar_cut_flag(u32 dev_id)
{
    struct devdrv_ctrl *ctrl = NULL;
    struct devdrv_pci_ctrl *pci_ctrl = NULL;

    ctrl = devdrv_get_top_half_devctrl_by_id(dev_id);
    if (ctrl == NULL) {
        devdrv_err("dev_id parameter is error, dev_id:%d.\n", dev_id);
        return -EINVAL;
    }

    pci_ctrl = (struct devdrv_pci_ctrl *)ctrl->priv;
    if (pci_ctrl == NULL) {
        devdrv_err("ctrl->priv parameter is error, dev_id:%d.\n", dev_id);
        return -EINVAL;
    }

    return pci_ctrl->device_bar_cut_flag;
}
EXPORT_SYMBOL(get_device_bar_cut_flag);

int devdrv_write_bar45(u32 devid, enum devdrv_addr_type type, u32 index, u64 offset, u64 val)
{
    void __iomem *addr_base = NULL;
    u64 addr = 0;
    size_t size = 0;
    int ret = 0;

    ret = devdrv_get_addr_info(devid, type, index, &addr, &size);
    if (ret != 0) {
        devdrv_info("get bar addr fail\n");
        return -EINVAL;
    }
    if (offset > (size - sizeof(u64)) || (offset % 0x4 != 0)) {
        devdrv_info("offset is more than size or is not 4 times\n");
        return -EINVAL;
    }
    addr_base = ioremap_wc(addr, size);
    if (addr_base == NULL) {
        devdrv_info("ioremap_wc failed\n");
        return -EINVAL;
    }

    writeq(val, (addr_base + offset)); //lint !e144

    iounmap(addr_base);

    return 0;
}
EXPORT_SYMBOL(devdrv_write_bar45);

int devdrv_read_bar45(u32 devid, enum devdrv_addr_type type, u32 index, u64 offset, u64 *val)
{
    void __iomem *addr_base = NULL;
    u64 addr = 0;
    size_t size = 0;
    int ret = 0;

    if (val == NULL) {
        devdrv_info("val is null\n");
        return -EINVAL;
    }

    ret = devdrv_get_addr_info(devid, type, index, &addr, &size);
    if (ret != 0) {
        devdrv_info("get bar addr fail\n");
        return -EINVAL;
    }

    if (offset > (size - sizeof(u64)) || (offset % 0x4 != 0)) {
        devdrv_info("offset is more than size or is not 4 times\n");
        return -EINVAL;
    }

    addr_base = ioremap_wc(addr, size);
    if (addr_base == NULL) {
        devdrv_info("ioremap_wc failed\n");
        return -EINVAL;
    }

    *val = readq(addr_base + offset);
    iounmap(addr_base);
    if (*val == BAR_EXCEP_VAL_64_BIT) {
        devdrv_err("*val=0x%llx, error!\n", *val);
        return -EINVAL;
    }

    return 0;
}
EXPORT_SYMBOL(devdrv_read_bar45);

int devdrv_get_irq_no(u32 devid, u32 index, unsigned int *irq)
{
    if (irq == NULL) {
        devdrv_info("null ptr\n");
        return -EINVAL;
    }
    *irq = index;
    return 0;
}
EXPORT_SYMBOL(devdrv_get_irq_no);

int devdrv_get_ts_drv_irq_vector_id(u32 devid, u32 index, unsigned int *entry)
{
    struct devdrv_ctrl *ctrl = NULL;
    struct devdrv_pci_ctrl *pci_ctrl = NULL;
    u32 id;
    int i;

    if (entry == NULL) {
        devdrv_info("devid %d null ptr\n", devid);
        return -EINVAL;
    }

    ctrl = devdrv_get_bottom_half_devctrl_by_id(devid);
    if ((ctrl == NULL) || (ctrl->priv == NULL)) {
        devdrv_info("devid %d parameter is error\n", devid);
        return -EINVAL;
    }
    pci_ctrl = (struct devdrv_pci_ctrl *)ctrl->priv;

    id = 0;
    for (i = 0; i < pci_ctrl->msix_irq_num; i++) {
        if (devdrv_is_tsdrv_irq(i) == true) {
            if (id == index) {
                *entry = i;
                return 0;
            }

            id++;
        }
    }

    return -EINVAL;
}
EXPORT_SYMBOL(devdrv_get_ts_drv_irq_vector_id);

int devdrv_get_ts_drv_irq_vector(u32 devid, u32 entry, unsigned int *irq)
{
    struct devdrv_ctrl *ctrl = NULL;
    struct devdrv_pci_ctrl *pci_ctrl = NULL;

    if (irq == NULL) {
        devdrv_info("devid %d null ptr\n", devid);
        return -EINVAL;
    }

    ctrl = devdrv_get_bottom_half_devctrl_by_id(devid);
    if ((ctrl == NULL) || (ctrl->priv == NULL)) {
        devdrv_info("devid %d parameter is error\n", devid);
        return -EINVAL;
    }
    pci_ctrl = (struct devdrv_pci_ctrl *)ctrl->priv;

    if (devdrv_is_tsdrv_irq(entry) == false) {
        devdrv_err("devid %d entry %d is error\n", devid, entry);
        return -EINVAL;
    }

    *irq = entry;

    return 0;
}
EXPORT_SYMBOL(devdrv_get_ts_drv_irq_vector);

int devdrv_register_irq_func_expand(u32 dev_id, int vector_index, irqreturn_t (*callback_func)(int, void *), void *para,
    const char *name)
{
    struct devdrv_ctrl *ctrl = NULL;
    struct devdrv_pci_ctrl *pci_ctrl = NULL;
    int ret;

    ctrl = devdrv_get_bottom_half_devctrl_by_id(dev_id);
    if (ctrl == NULL) {
        devdrv_err("dev id %d parameter is error\n", dev_id);
        return -EINVAL;
    }
    pci_ctrl = (struct devdrv_pci_ctrl *)ctrl->priv;

    ret = devdrv_register_irq_func((void *)pci_ctrl, vector_index, callback_func, para, name);
    if (ret) {
        devdrv_err("devdrv_register_irq_funcr is error, ret = %d\n", ret);
        return -EINVAL;
    }

    return 0;
}
EXPORT_SYMBOL(devdrv_register_irq_func_expand);

int devdrv_unregister_irq_func_expand(u32 dev_id, int vector_index, void *para)
{
    struct devdrv_ctrl *ctrl = NULL;
    struct devdrv_pci_ctrl *pci_ctrl = NULL;
    int ret;

    ctrl = devdrv_get_bottom_half_devctrl_by_id(dev_id);
    if (ctrl == NULL) {
        devdrv_err("dev id %d parameter is error\n", dev_id);
        return -EINVAL;
    }
    pci_ctrl = (struct devdrv_pci_ctrl *)ctrl->priv;

    ret = devdrv_unregister_irq_func((void *)pci_ctrl, vector_index, para);
    if (ret) {
        devdrv_err("devdrv_unregister_irq_funcr is error, ret = %d\n", ret);
        return -EINVAL;
    }

    return 0;
}
EXPORT_SYMBOL(devdrv_unregister_irq_func_expand);

int devdrv_get_pci_dev_info(u32 devid, struct devdrv_pci_dev_info *dev_info)
{
    struct devdrv_ctrl *ctrl = NULL;
    struct devdrv_pci_ctrl *pci_ctrl = NULL;

    if (dev_info == NULL) {
        devdrv_info("null ptr\n");
        return -EINVAL;
    }

    ctrl = devdrv_get_top_half_devctrl_by_id(devid);
    if (ctrl == NULL) {
        devdrv_info("pci dev devid parameter is error\n");
        return -EINVAL;
    }
    pci_ctrl = (struct devdrv_pci_ctrl *)ctrl->priv;
    dev_info->bus_no = (u8)pci_ctrl->pdev->bus->number;
    dev_info->device_no = (u8)((pci_ctrl->pdev->devfn >> 3) & 0x1F);
    dev_info->function_no = (u8)(pci_ctrl->pdev->devfn & 0x7);

    return 0;
}
EXPORT_SYMBOL(devdrv_get_pci_dev_info);

int devdrv_get_pcie_id_info(u32 devid, struct devdrv_pcie_id_info *dev_info)
{
    struct devdrv_ctrl *ctrl = NULL;
    struct devdrv_pci_ctrl *pci_ctrl = NULL;

    if (dev_info == NULL) {
        devdrv_info("null ptr\n");
        return -EINVAL;
    }

    ctrl = devdrv_get_top_half_devctrl_by_id(devid);
    if (ctrl == NULL) {
        devdrv_info("pcie devid parameter is error\n");
        return -EINVAL;
    }
    pci_ctrl = (struct devdrv_pci_ctrl *)ctrl->priv;
    dev_info->venderid = pci_ctrl->pdev->vendor;
    dev_info->subvenderid = pci_ctrl->pdev->subsystem_vendor;
    dev_info->deviceid = pci_ctrl->pdev->device;
    dev_info->subdeviceid = pci_ctrl->pdev->subsystem_device;
    dev_info->bus = pci_ctrl->pdev->bus->number;
    dev_info->device = (pci_ctrl->pdev->devfn >> 3) & 0x1F;
    dev_info->fn = (pci_ctrl->pdev->devfn) & 0x7;

    return 0;
}
EXPORT_SYMBOL(devdrv_get_pcie_id_info);

int devdrv_get_atu_info(u32 devid, int atu_type, struct devdrv_iob_atu **atu, u64 *host_phy_base)
{
    struct devdrv_pci_ctrl *pci_ctrl = NULL;
    struct devdrv_ctrl *ctrl = NULL;

    if (devid >= MAX_DEV_CNT) {
        devdrv_err("dev %d is error.\n", devid);
        return -EINVAL;
    }

    ctrl = devdrv_get_bottom_half_devctrl_by_id(devid);
    if (ctrl == NULL) {
        devdrv_err("get devctrl by id failed, dev_id: %d\n", devid);
        return -EINVAL;
    }
    pci_ctrl = (struct devdrv_pci_ctrl *)ctrl->priv;

    switch (atu_type) {
        case ATU_TYPE_RX_MEM:
            *atu = pci_ctrl->mem_rx_atu;
            *host_phy_base = pci_ctrl->mem_phy_base;
            break;
        default:
            devdrv_err("dev %d atu_type %d not surport.\n", devid, atu_type);
            return -EINVAL;
    }

    return 0;
}

/* async DMA link prepare */
struct devdrv_dma_prepare *devdrv_dma_link_prepare(u32 devid, enum devdrv_dma_data_type type,
    struct devdrv_dma_node *dma_node, u32 node_cnt)
{
    struct devdrv_dma_sq_node *current_sq_node = NULL;
    struct devdrv_dma_prepare *dma_prepare = NULL;
    struct devdrv_ctrl *drv_ctrl = NULL;
    struct device *dev = NULL;
    u32 size;
    u32 i;

    if (devid >= MAX_DEV_CNT) {
        devdrv_err("invalid dev id, id overflow!\n");
        return NULL;
    }
    if (dma_node == NULL) {
        devdrv_err("dma_node is null\n");
        return NULL;
    }
    if (node_cnt == 0) {
        devdrv_err("sq_dma_addr is 0\n");
        return NULL;
    }

    drv_ctrl = devdrv_get_bottom_half_devctrl_by_id(devid);

    if (drv_ctrl == NULL) {
        devdrv_err("drv_ctrl is null\n");
        return NULL;
    }
    dev = drv_ctrl->dev;

    dma_prepare = kzalloc(sizeof(struct devdrv_dma_prepare), GFP_KERNEL | __GFP_ACCOUNT);
    if (dma_prepare == NULL) {
        devdrv_err("%s, dma_prepare alloc fail\n", dev_driver_string(dev));
        return NULL;
    }

    dma_prepare->devid = devid;

    size = (node_cnt + 2) * sizeof(struct devdrv_dma_sq_node);
    dma_prepare->sq_size = size;
    dma_prepare->sq_base = dma_alloc_coherent(dev, dma_prepare->sq_size, &dma_prepare->sq_dma_addr,
        GFP_KERNEL | __GFP_ACCOUNT | __GFP_NOWARN);
    if (dma_prepare->sq_base == NULL) {
        dma_prepare->sq_base = dma_alloc_coherent(dev, dma_prepare->sq_size, &dma_prepare->sq_dma_addr,
            GFP_DMA32 | GFP_KERNEL | __GFP_ACCOUNT | __GFP_NOWARN);
        if (dma_prepare->sq_base == NULL) {
            devdrv_warn("%s, dma sq alloc fail\n", dev_driver_string(dev));
            goto free_prepare;
        }
    }

    size = (node_cnt + 2) * sizeof(struct devdrv_dma_cq_node);

    dma_prepare->cq_size = size;
    dma_prepare->cq_base = dma_alloc_coherent(dev, dma_prepare->cq_size, &dma_prepare->cq_dma_addr,
        GFP_KERNEL | __GFP_ACCOUNT | __GFP_NOWARN);
    if (dma_prepare->cq_base == NULL) {
        dma_prepare->cq_base = dma_alloc_coherent(dev, dma_prepare->cq_size, &dma_prepare->cq_dma_addr,
            GFP_DMA32 | GFP_KERNEL | __GFP_ACCOUNT | __GFP_NOWARN);
        if (dma_prepare->cq_base == NULL) {
            devdrv_warn("%s, dma cq alloc fail\n", dev_driver_string(dev));
            goto free_sq;
        }
    }

    for (i = 0; i < node_cnt; i++) {
        current_sq_node = (struct devdrv_dma_sq_node *)(dma_prepare->sq_base) + i;
        current_sq_node->src_addr_l = dma_node[i].src_addr & 0xFFFFFFFF;
        current_sq_node->src_addr_h = ((u64)dma_node[i].src_addr) >> 32;
        current_sq_node->dst_addr_l = dma_node[i].dst_addr & 0xFFFFFFFF;
        current_sq_node->dst_addr_h = ((u64)dma_node[i].dst_addr) >> 32;
        current_sq_node->length = dma_node[i].size;

        if (dma_node[i].direction == DEVDRV_DMA_DEVICE_TO_HOST) {
            current_sq_node->opcode = DEVDRV_DMA_WRITE;
        } else if (dma_node[i].direction == DEVDRV_DMA_HOST_TO_DEVICE) {
            current_sq_node->opcode = DEVDRV_DMA_READ;
        } else {
            devdrv_err("%s, DMA direction para is invalid\n", dev_driver_string(dev));
            goto free_cq;
        }
        current_sq_node->pf = DEVDRV_PF_NUM;
        if (i == node_cnt - 1) {
            current_sq_node->ldie = 1;
        }
        /* enable RO */
        current_sq_node->attr = DEVDRV_DMA_SQ_ATTR_RO;
    }

    return dma_prepare;

free_cq:
    dma_free_coherent(dev, dma_prepare->cq_size, dma_prepare->cq_base, dma_prepare->cq_dma_addr);
free_sq:
    dma_free_coherent(dev, dma_prepare->sq_size, dma_prepare->sq_base, dma_prepare->sq_dma_addr);
free_prepare:
    kfree(dma_prepare);
    dma_prepare = NULL;
    return NULL;
}
EXPORT_SYMBOL(devdrv_dma_link_prepare);

/* async DMA link release */
int devdrv_dma_link_free(struct devdrv_dma_prepare *dma_prepare)
{
    struct devdrv_ctrl *drv_ctrl = NULL;
    struct device *dev = NULL;

    if (dma_prepare == NULL) {
        devdrv_err("dma_prepare is null\n");
        return -EINVAL;
    }

    drv_ctrl = devdrv_get_bottom_half_devctrl_by_id(dma_prepare->devid);
    if (drv_ctrl == NULL) {
        devdrv_err("drv_ctrl is null\n");
        return -EINVAL;
    }

    dev = drv_ctrl->dev;
    dma_free_coherent(dev, dma_prepare->cq_size, dma_prepare->cq_base, dma_prepare->cq_dma_addr);
    dma_free_coherent(dev, dma_prepare->sq_size, dma_prepare->sq_base, dma_prepare->sq_dma_addr);
    kfree(dma_prepare);
    dma_prepare = NULL;
    return 0;
}
EXPORT_SYMBOL(devdrv_dma_link_free);

static const struct devdrv_ops ops = {
    .alloc_trans_chan = devdrv_alloc_trans_queue,
    .realease_trans_chan = devdrv_free_trans_queue,
    .alloc_non_trans_chan = devdrv_alloc_non_trans_queue,
    .release_non_trans_chan = devdrv_free_non_trans_queue
};

int devdrv_register_pci_devctrl(struct devdrv_pci_ctrl *pci_ctrl)
{
    struct devdrv_ctrl pci_dev_ctrl;
    int dev_id;

    pci_dev_ctrl.priv = pci_ctrl;
    pci_dev_ctrl.dev_type = DEV_TYPE_PCI;
    pci_dev_ctrl.dev = &pci_ctrl->pdev->dev;
    pci_dev_ctrl.bus = pci_ctrl->pdev->bus;
    pci_dev_ctrl.pdev = pci_ctrl->pdev;
    dev_id = devdrv_slave_dev_add(&pci_dev_ctrl);
    devdrv_info("%s, dev_id:%d, bus:%pK\n", dev_driver_string(&pci_ctrl->pdev->dev), dev_id, pci_dev_ctrl.bus);

    if (dev_id < 0) {
        devdrv_err("%s, pci device register fail\n", dev_driver_string(&pci_ctrl->pdev->dev));
        return -ENOSPC;
    } else {
        pci_ctrl->dev_id = (u32)dev_id;
        return 0;
    }
}

void devdrv_register_half_devctrl(struct devdrv_pci_ctrl *pci_ctrl)
{
    u32 dev_id;

    dev_id = pci_ctrl->dev_id;
    ctrls[dev_id].ops = &ops;
    ctrls[dev_id].startup_flg = DEVDRV_DEV_STARTUP_BOTTOM_HALF_OK;
}

int devdrv_dev_online(struct devdrv_pci_ctrl *pci_ctrl)
{
    struct devdrv_ctrl pci_dev_ctrl;
    struct devdrv_pci_ctrl *ctrl = NULL;
    int ret;
    int i;

    devdrv_info("dev_id %d online chip_type %d\n", pci_ctrl->dev_id, pci_ctrl->shr_para->chip_type);

    mutex_lock(&devdrv_ctrl_mutex);

    /* register this pci device to devdrv_ctrls */
    pci_dev_ctrl.dev_id = pci_ctrl->dev_id;

    ret = devdrv_call_init_instance(&pci_dev_ctrl);
    if (ret != 0) {
        mutex_unlock(&devdrv_ctrl_mutex);
        devdrv_err("dev_id %d pci device register fail\n", pci_ctrl->dev_id);
        return -ENOSPC;
    }
    devdrv_debug("chip_type %d\n", pci_ctrl->shr_para->chip_type);
    if (pci_ctrl->shr_para->chip_type == (int)BAR_EXCEP_VAL_32_BIT) {
        mutex_unlock(&devdrv_ctrl_mutex);
        devdrv_err("dev_id %u, chip_type %d, error\n", pci_ctrl->dev_id, pci_ctrl->shr_para->chip_type);
        return -ENODEV;
    }
    if (pci_ctrl->shr_para->chip_type == HISI_MINI) {
        mutex_unlock(&devdrv_ctrl_mutex);
        return 0;
    }

    /* p2p */
    if (pci_ctrl->dev_id >= DEVDRV_P2P_SURPORT_MAX_DEVICE) {
        mutex_unlock(&devdrv_ctrl_mutex);
        devdrv_info("dev %d, only surport %d device for p2p\n", pci_ctrl->dev_id, DEVDRV_P2P_SURPORT_MAX_DEVICE);
        return 0;
    }

    /* config tx atu & p2p msg base */
    for (i = 0; i < DEVDRV_P2P_SURPORT_MAX_DEVICE; i++) {
        if (ctrls[i].startup_flg != DEVDRV_DEV_STARTUP_BOTTOM_HALF_OK)
            continue;

        ctrl = (struct devdrv_pci_ctrl *)ctrls[i].priv;

        if (pci_ctrl->dev_id == ctrl->dev_id)
            continue;

        /* Configure the bar information of the newly added device to the device that has been scanned. */
        /* mem bar */
        (void)devdrv_set_tx_atu(ctrl->io_base, ctrl->dev_id, pci_ctrl->dev_id, ATU_TYPE_TX_MEM,
            (u64)pci_ctrl->mem_phy_base, pci_ctrl->mem_phy_size, ctrl->shr_para);

        /* io bar */
        (void)devdrv_set_tx_atu(ctrl->io_base, ctrl->dev_id, pci_ctrl->dev_id, ATU_TYPE_TX_IO,
            (u64)pci_ctrl->io_phy_base, pci_ctrl->io_phy_size, ctrl->shr_para);

        ctrl->shr_para->p2p_msg_base_addr[pci_ctrl->dev_id] =
            (u64)pci_ctrl->rsv_mem_phy_base + DEVDRV_RESERVE_MEM_MSG_OFFSET;
        ctrl->shr_para->p2p_db_base_addr[pci_ctrl->dev_id] =
            (u64)pci_ctrl->rsv_mem_phy_base + DEVDRV_RESERVE_MEM_DB_BASE;

        /* notice device */
        (void)devdrv_notify_dev_online(ctrl->msg_dev, pci_ctrl->dev_id, DEVDRV_DEV_ONLINE);

        /* mem bar */
        (void)devdrv_set_tx_atu(pci_ctrl->io_base, pci_ctrl->dev_id, ctrl->dev_id,
            ATU_TYPE_TX_MEM, (u64)ctrl->mem_phy_base, ctrl->mem_phy_size, pci_ctrl->shr_para);

        /* io bar */
        (void)devdrv_set_tx_atu(pci_ctrl->io_base, pci_ctrl->dev_id, ctrl->dev_id,
            ATU_TYPE_TX_IO, (u64)ctrl->io_phy_base, ctrl->io_phy_size, pci_ctrl->shr_para);

        pci_ctrl->shr_para->p2p_msg_base_addr[ctrl->dev_id] =
            (u64)ctrl->rsv_mem_phy_base + DEVDRV_RESERVE_MEM_MSG_OFFSET;
        pci_ctrl->shr_para->p2p_db_base_addr[ctrl->dev_id] = (u64)ctrl->rsv_mem_phy_base + DEVDRV_RESERVE_MEM_DB_BASE;
    }

    (void)devdrv_notify_dev_online(pci_ctrl->msg_dev, pci_ctrl->dev_id, DEVDRV_DEV_ONLINE);

    mutex_unlock(&devdrv_ctrl_mutex);

    return 0;
}

int devdrv_pcie_prereset(u32 dev_id)
{
    struct devdrv_pci_ctrl *pci_ctrl = NULL;
    struct devdrv_ctrl *dev_ctrl = NULL;

    if (dev_id >= MAX_DEV_CNT) {
        devdrv_err(" dev id %u not exist !\n", dev_id);
        return -EINVAL;
    }

    dev_ctrl = devdrv_get_bottom_half_devctrl_by_id(dev_id);
    if (dev_ctrl == NULL) {
        devdrv_err("get devctrl by id failed, dev_id: %u\n", dev_id);
        return -EINVAL;
    }
    devdrv_info("call pcie_prereset, dev_id: %u\n", dev_id);
    pci_ctrl = (struct devdrv_pci_ctrl *)dev_ctrl->priv;
    devdrv_pci_stop_and_remove_bus_device_locked(pci_ctrl->pdev);

    return 0;
}
EXPORT_SYMBOL(devdrv_pcie_prereset);

void devdrv_set_device_boot_finish(struct devdrv_pci_ctrl *pci_ctrl)
{
    struct devdrv_startup_status *startup_status = &(pci_ctrl->startup_status);

    if ((startup_status->status == DEVDRV_STARTUP_STATUS_FINISH) &&
        ((startup_status->module_bit_map & DEVDRV_HOST_MODULE_MASK) == DEVDRV_HOST_MODULE_MASK)) {
        /* set device boot status:boot finish */
        devdrv_set_device_boot_status(pci_ctrl, DSMI_BOOT_STATUS_FINISH);
        devdrv_info("dev id %d boot finish\r\n", pci_ctrl->dev_id);
    }
}

void devdrv_set_startup_status(struct devdrv_pci_ctrl *pci_ctrl, int status)
{
    struct devdrv_startup_status *startup_status = &(pci_ctrl->startup_status);

    if (status == DEVDRV_STARTUP_STATUS_INIT) {
        startup_status->module_bit_map = 0;
        devdrv_info("dev id %d startup status init jiffies %ld \r\n", pci_ctrl->dev_id, jiffies);
    } else {
        devdrv_info("dev id %d pre status %d, current %d, use time %d(ms) \r\n", pci_ctrl->dev_id,
            startup_status->status, status, jiffies_to_msecs(jiffies - startup_status->timestamp));
    }

    startup_status->status = status;
    startup_status->timestamp = jiffies;

    devdrv_set_device_boot_finish(pci_ctrl);
}

bool devdrv_can_hotreset(struct devdrv_startup_status *startup_status)
{
    bool ret = false;

    mutex_lock(&devdrv_ctrl_mutex);

    if (startup_status->status == DEVDRV_STARTUP_STATUS_FINISH) {
        if ((startup_status->module_bit_map & DEVDRV_HOST_MODULE_MASK) == DEVDRV_HOST_MODULE_MASK) {
            /* all module init finish */
            ret = true;
        } else if (jiffies_to_msecs(jiffies - startup_status->timestamp) > DEVDRV_MODULE_FINISH_TIMEOUT) {
            /* module init timeout */
            ret = true;
        }
    } else if (startup_status->status == DEVDRV_STARTUP_STATUS_TIMEOUT) {
        /* load timeout */
        ret = true;
    } else {
        if (jiffies_to_msecs(jiffies - startup_status->timestamp) > DEVDRV_MODULE_INIT_TIMEOUT) {
            /* slave chip no load */
            ret = true;
        }
    }

    mutex_unlock(&devdrv_ctrl_mutex);

    devdrv_info("status %d, module_bit_map 0x%x, timestamp %ld cur %ld ret %d \r\n", startup_status->status,
        startup_status->module_bit_map, startup_status->timestamp, jiffies, ret);

    return ret;
}

struct pci_dev *devdrv_get_device_pf(struct pci_dev *pdev, unsigned int pf_num)
{
    unsigned int devfn;
    int domain_nr;
    struct pci_dev *dev = NULL;

    devfn = PCI_DEVFN(PCI_SLOT(pdev->devfn), pf_num);
    domain_nr = pci_domain_nr(pdev->bus);
    dev = pci_get_domain_bus_and_slot(domain_nr, pdev->bus->number, devfn);

    return dev;
}

void devdrv_device_unbind_driver(struct pci_dev *pdev_b)
{
    struct pci_dev *pdev = NULL;

#ifdef CFG_SOC_PLATFORM_CLOUD
    if ((u32)PCI_FUNC(pdev_b->devfn) == PCIE_PF_NUM) {
        /* network pf0 */
        pdev = devdrv_get_device_pf(pdev_b, NETWORK_PF_0);
        if (pdev != NULL) {
            devdrv_info("pcie hotreset pci dev(%02x:%02x.%d), unbind driver\n", pdev->bus->number,
                PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
            device_release_driver(&pdev->dev);
        }

        /* network pf1 */
        pdev = devdrv_get_device_pf(pdev_b, NETWORK_PF_1);
        if (pdev != NULL) {
            devdrv_info("pcie hotreset pci dev(%02x:%02x.%d), unbind driver\n", pdev->bus->number,
                PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
            device_release_driver(&pdev->dev);
        }
    }
#endif

    /* davinci pf */
    pdev = pdev_b;
    devdrv_info("pcie hotreset pci dev(%02x:%02x.%d), unbind driver\n", pdev->bus->number, PCI_SLOT(pdev->devfn),
        PCI_FUNC(pdev->devfn));
    device_release_driver(&pdev->dev);
}

void devdrv_device_bus_rescan(struct pci_dev *pdev)
{
    int num;
    num = devdrv_pci_rescan_bus_locked(pdev->bus);
    devdrv_err("pcie hotreset pci bus dev(%02x:%02x.%d), rescan device device num %d\n", pdev->bus->number,
        PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn), num);
}

void devdrv_device_bus_remove(struct pci_dev *pdev_r)
{
    struct pci_dev *pdev = NULL;

#ifdef CFG_SOC_PLATFORM_CLOUD
    u8 func_num = PCI_FUNC(pdev_r->devfn);
    if ((u32)func_num == PCIE_PF_NUM) {
        /* network pf0 */
        pdev = devdrv_get_device_pf(pdev_r, NETWORK_PF_0);
        if (pdev != NULL) {
            devdrv_info("pcie hotreset pci dev(%02x:%02x.%d), bus remove\n", pdev->bus->number, PCI_SLOT(pdev->devfn),
                PCI_FUNC(pdev->devfn));
            devdrv_pci_stop_and_remove_bus_device_locked(pdev);
        }

        /* network pf1 */
        pdev = devdrv_get_device_pf(pdev_r, NETWORK_PF_1);
        if (pdev != NULL) {
            devdrv_info("pcie hotreset pci dev(%02x:%02x.%d), bus remove\n", pdev->bus->number, PCI_SLOT(pdev->devfn),
                PCI_FUNC(pdev->devfn));
            devdrv_pci_stop_and_remove_bus_device_locked(pdev);
        }
    }
#endif

    /* davinci pf */
    pdev = pdev_r;
    devdrv_info("pcie hotreset pci dev(%02x:%02x.%d), bus remove\n", pdev->bus->number, PCI_SLOT(pdev->devfn),
        PCI_FUNC(pdev->devfn));

    devdrv_pci_stop_and_remove_bus_device_locked(pdev);
}

void devdrv_hotreset_set_timestamp(struct devdrv_ctrl *dev_ctrl)
{
    dev_ctrl->timestamp = (unsigned long)jiffies;
}

unsigned long devdrv_hotreset_get_timetamp(struct devdrv_ctrl *dev_ctrl)
{
    return dev_ctrl->timestamp;
}

bool devdrv_pre_hotreset_check(int dev_num)
{
    struct devdrv_ctrl *dev_ctrl = NULL;
    struct devdrv_pci_ctrl *pci_ctrl = NULL;
    struct devdrv_startup_status *startup_status = NULL;
    int i, ready_dev_num;
    int ret = false;

    ready_dev_num = 0;

    for (i = 0; i < BOARD_CLOUD_AI_SERVER_DEV_NUM * MAX_OS_EACH_AISERVER; i++) {
        dev_ctrl = &ctrls[i];

        pci_ctrl = (struct devdrv_pci_ctrl *)dev_ctrl->priv;
        if (pci_ctrl != NULL) {
            startup_status = &(pci_ctrl->startup_status);
            if (devdrv_can_hotreset(startup_status)) {
                ready_dev_num++;
            } else {
                devdrv_info("dev id %d cannot hotreset. it's status %d\r\n", dev_ctrl->dev_id, startup_status->status);
            }
        }
    }

    devdrv_info("dev num %d ready_dev_num %d\r\n", dev_num, ready_dev_num);

    if (ready_dev_num == dev_num) {
        ret = true;
    }

    return ret;
}

void devdrv_before_hot_reset(void)
{
    struct devdrv_ctrl *dev_ctrl = NULL;
    struct devdrv_pci_ctrl *pci_ctrl = NULL;
    int i;

#ifdef CFG_SOC_PLATFORM_CLOUD
    for (i = 0; i < BOARD_CLOUD_AI_SERVER_DEV_NUM * MAX_OS_EACH_AISERVER; i++) {
        dev_ctrl = &ctrls[i];

        if (dev_ctrl->startup_flg != DEVDRV_DEV_STARTUP_UNPROBED) {
            pci_ctrl = (struct devdrv_pci_ctrl *)dev_ctrl->priv;
            if (pci_ctrl->shr_para->chip_id > 0) {
                devdrv_info("pcie hotreset dev %d before hot reset remove\r\n", dev_ctrl->dev_id);
                dev_ctrl->master_flag = 0;
                devdrv_device_bus_remove(dev_ctrl->pdev);
                msleep(1); /* free cpu */
            }
        }
    }

    for (i = 0; i < BOARD_CLOUD_AI_SERVER_DEV_NUM * MAX_OS_EACH_AISERVER; i++) {
        dev_ctrl = &ctrls[i];

        if (dev_ctrl->startup_flg != DEVDRV_DEV_STARTUP_UNPROBED) {
            pci_ctrl = (struct devdrv_pci_ctrl *)dev_ctrl->priv;
            if (pci_ctrl->shr_para->chip_id == 0) {
                devdrv_info("pcie hotreset dev %d before hot reset unbind\r\n", dev_ctrl->dev_id);
                dev_ctrl->master_flag = 1;
                pci_ctrl->shr_para->hot_reset_pcie_flag = DEVDRV_PCIE_HOT_RESET_FLAG;
                devdrv_device_unbind_driver(dev_ctrl->pdev);
                msleep(1); /* free cpu */
            }
        }
    }
#else
    for (i = 0; i < MAX_DEV_CNT; i++) {
        dev_ctrl = &ctrls[i];
        if (dev_ctrl == NULL) {
            continue;
        }
        if (dev_ctrl->startup_flg != DEVDRV_DEV_STARTUP_UNPROBED) {
            pci_ctrl = (struct devdrv_pci_ctrl *)dev_ctrl->priv;
            devdrv_info("pcie hotreset dev %d before hot reset unbind\r\n", dev_ctrl->dev_id);
            devdrv_device_unbind_driver(dev_ctrl->pdev);
            msleep(1); /* free cpu */
        }
    }
#endif
}

void devdrv_pci_hot_reset(struct devdrv_ctrl *dev_ctrl)
{
    int ret;

    ret = pci_try_reset_function(dev_ctrl->pdev);
    devdrv_info("devid %d reset function ret %d\n", dev_ctrl->dev_id, ret);

    return;
}

void devdrv_do_hot_reset(void)
{
    struct devdrv_ctrl *dev_ctrl = NULL;
    int i;
#ifdef CFG_SOC_PLATFORM_CLOUD
    for (i = 0; i < BOARD_CLOUD_AI_SERVER_DEV_NUM * MAX_OS_EACH_AISERVER; i++) {
        dev_ctrl = &ctrls[i];

        if (dev_ctrl->master_flag == 1) {
            if (dev_ctrl->bus != NULL) {
                devdrv_info("pcie hotreset dev %d do hot reset\r\n", dev_ctrl->dev_id);
                devdrv_pci_hot_reset(dev_ctrl);
            }
        }
    }
#else
    for (i = 0; i < MAX_DEV_CNT; i++) {
        dev_ctrl = &ctrls[i];
        if (dev_ctrl == NULL) {
            continue;
        }

        if (dev_ctrl->bus != NULL) {
            devdrv_info("pcie hotreset dev %d do hot reset\r\n", dev_ctrl->dev_id);
            devdrv_pci_hot_reset(dev_ctrl);
        }
    }
#endif
}

void devdrv_after_hot_reset(int num)
{
    struct devdrv_ctrl *dev_ctrl = NULL;
    int delay_count = 0;
    int i;
    int num_after;
    int max_dev_num;
#ifdef CFG_SOC_PLATFORM_CLOUD
    max_dev_num = BOARD_CLOUD_AI_SERVER_DEV_NUM * MAX_OS_EACH_AISERVER;
    for (i = 0; i < max_dev_num; i++) {
        dev_ctrl = &ctrls[i];

        if (dev_ctrl->master_flag == 1) {
            dev_ctrl->master_flag = 0;
            if (dev_ctrl->pdev != NULL) {
                devdrv_info("pcie hotreset dev %d remove\r\n", dev_ctrl->dev_id);
                devdrv_device_bus_remove(dev_ctrl->pdev);
                msleep(1); /* free cpu */
            }
        }
    }
#else
    max_dev_num = MAX_DEV_CNT;
    for (i = 0; i < max_dev_num; i++) {
        dev_ctrl = &ctrls[i];
        if (dev_ctrl == NULL) {
            continue;
        }

        if (dev_ctrl->pdev != NULL) {
            devdrv_info("pcie hotreset dev %d remove\r\n", dev_ctrl->dev_id);
            devdrv_device_bus_remove(dev_ctrl->pdev);
            msleep(1); /* free cpu */
        }
    }
#endif

    for (i = 0; i < max_dev_num; i++) {
        dev_ctrl = &ctrls[i];
        devdrv_hotreset_set_timestamp(dev_ctrl);
    }
    /* make sure rescan device num equal with num before hot reset */
    do {
        for (i = 0; i < max_dev_num; i++) {
            dev_ctrl = &ctrls[i];
            if (dev_ctrl == NULL) {
                continue;
            }
            devdrv_hotreset_set_timestamp(dev_ctrl);
            if (dev_ctrl->bus != NULL) {
                devdrv_info("pcie hotreset dev %d after hot reset\r\n", dev_ctrl->dev_id);
                devdrv_device_bus_rescan(dev_ctrl->bus->self);
            }
        }
        devdrv_init_dev_num();
        num_after = devdrv_get_dev_num();
        if (num == num_after) {
            break;
        }
        delay_count++;
        ssleep(DEVDRV_HOT_RESET_DELAY);
    } while (delay_count < DEVDRV_MAX_DELAY_COUNT);

    devdrv_info("rescan cost %d time, num_before %d, num_after %d\n", delay_count, num, num_after);
}

#ifdef CFG_SOC_PLATFORM_CLOUD
bool devdrv_is_amp_system(u32 dev_id, int node_id, int chip_id)
{
    struct devdrv_ctrl *ctrl = NULL;
    struct devdrv_pci_ctrl *pci_ctrl = NULL;
    struct devdrv_shr_para __iomem *shr_para = NULL;
    int i;

    if (chip_id != 0) {
        return false;
    }

    for (i = 0; i < MAX_DEV_CNT; i++) {
        if (dev_id == i) {
            continue;
        }
        ctrl = devdrv_get_bottom_half_devctrl_by_id(i);
        if ((ctrl == NULL) || (ctrl->priv == NULL)) {
            continue;
        }
        pci_ctrl = (struct devdrv_pci_ctrl *)ctrl->priv;
        shr_para = pci_ctrl->shr_para;
        if ((shr_para->node_id == node_id) && (shr_para->chip_id != 0)) {
            return false;
        }
    }

    return true;
}
#endif

int devdrv_hot_reset_single_device(u32 dev_id)
{
    struct devdrv_ctrl *dev_ctrl = NULL;
    struct devdrv_pci_ctrl *pci_ctrl = NULL;
    int delay_count = 0;

    dev_ctrl = &ctrls[dev_id];
    pci_ctrl = (struct devdrv_pci_ctrl *)dev_ctrl->priv;
    if (pci_ctrl == NULL) {
        devdrv_info("dev id %d is not init\r\n", dev_id);
        return -EINVAL;
    }

#ifdef CFG_SOC_PLATFORM_CLOUD
    struct devdrv_startup_status *startup_status = NULL;

    startup_status = &(pci_ctrl->startup_status);
    if (devdrv_can_hotreset(startup_status) == false) {
        devdrv_info("dev id %d cannot hotreset. it's status %d\r\n", dev_id, startup_status->status);
        return -EAGAIN;
    }

    if (devdrv_is_amp_system(dev_id, pci_ctrl->shr_para->node_id, pci_ctrl->shr_para->chip_id) == false) {
        devdrv_info("dev id %d is not amp system\r\n", dev_id);
        return -EINVAL;
    }
#endif
    /* before hotreset: unbind driver */
    devdrv_device_unbind_driver(dev_ctrl->pdev);

    /* do hotreset */
    devdrv_info("pcie hotreset dev %d do hot reset\r\n", dev_id);
    devdrv_pci_hot_reset(dev_ctrl);

    ssleep(3);

    /* after hotreset */
    devdrv_device_bus_remove(dev_ctrl->pdev);

    if (dev_ctrl->priv != NULL) {
        devdrv_info("pcie hotreset dev %d after bus remove\r\n", dev_id);
    }

    do {
        devdrv_hotreset_set_timestamp(dev_ctrl);
        devdrv_device_bus_rescan(dev_ctrl->bus->self);
        if (dev_ctrl->priv != NULL) {
            break;
        }
        ssleep(3);
    } while (delay_count < DEVDRV_MAX_DELAY_COUNT);

    return 0;
}

int devdrv_hot_reset_all_device(void)
{
    int num_before;

    num_before = devdrv_get_dev_num();
    if (num_before <= 0) {
        devdrv_err("pcie hotreset all device have %d device\n", num_before);
        return -EAGAIN;
    }

#ifdef CFG_SOC_PLATFORM_CLOUD
    if (devdrv_pre_hotreset_check(num_before) == false) {
        devdrv_err("can not hotreset device num %d\r\n", num_before);
        return -EAGAIN;
    }
#endif

    devdrv_before_hot_reset();

    devdrv_do_hot_reset();

    ssleep(DEVDRV_HOT_RESET_DELAY);

    devdrv_after_hot_reset(num_before);

    return 0;
}


int devdrv_hot_reset_device(u32 dev_id)
{
    int ret = 0;

    mutex_lock(&devdrv_ctrl_mutex);
    if (dev_id == 0xff) {
        if (g_devdrv_ctrl_hot_reset_status != 0) {
            mutex_unlock(&devdrv_ctrl_mutex);
            devdrv_err("dev %d pcie hotreset, but dev mask %lu on process\r\n", dev_id, g_devdrv_ctrl_hot_reset_status);
            return -EAGAIN;
        } else {
            g_devdrv_ctrl_hot_reset_status = DEVDRV_PCIE_HOT_RESET_ALL_DEVICE_MASK;
        }
    } else if (dev_id < MAX_DEV_CNT) {
        if ((g_devdrv_ctrl_hot_reset_status >> dev_id) & 1 == 1) {
            mutex_unlock(&devdrv_ctrl_mutex);
            devdrv_err("dev %d pcie hotreset, but dev mask %lu on process\r\n", dev_id, g_devdrv_ctrl_hot_reset_status);
            return -EAGAIN;
        } else {
            g_devdrv_ctrl_hot_reset_status |= 1 << dev_id;
        }
    } else {
        mutex_unlock(&devdrv_ctrl_mutex);
        devdrv_err("pcie hotreset dev %d err\r\n", dev_id);
        return -EINVAL;
    }
    mutex_unlock(&devdrv_ctrl_mutex);

    if (dev_id == 0xff) {
        ret = devdrv_hot_reset_all_device();
    } else {
        ret = devdrv_hot_reset_single_device(dev_id);
    }

    if (ret == 0) {
        devdrv_info("pcie hotreset dev %d hot reset success\r\n", dev_id);
    }

    mutex_lock(&devdrv_ctrl_mutex);
    if (dev_id == 0xff) {
        g_devdrv_ctrl_hot_reset_status = 0;
    } else {
        g_devdrv_ctrl_hot_reset_status &= ~(1 << dev_id);
    }
    mutex_unlock(&devdrv_ctrl_mutex);

    return ret;
}
EXPORT_SYMBOL(devdrv_hot_reset_device);

int devdrv_get_host_phy_mach_flag(u32 devid, u32 *host_flag)
{
    struct devdrv_pci_ctrl *pci_ctrl = NULL;
    struct devdrv_ctrl *ctrl = NULL;

    if (devid >= MAX_DEV_CNT) {
        devdrv_err("dev id is invalid\n");
        return -EINVAL;
    }
    if (host_flag == NULL) {
        devdrv_err("host_flag is null\n");
        return -EINVAL;
    }
    ctrl = devdrv_get_bottom_half_devctrl_by_id(devid);
    if (ctrl == NULL) {
        devdrv_err("get devctrl by id err:%d!\n", devid);
        return -EINVAL;
    }

    pci_ctrl = ctrl->priv;
#ifndef DRV_UT
    if (pci_ctrl == NULL) {
        devdrv_err("pci_ctrl is null!\n");
        return -EINVAL;
    }
#endif
    *host_flag = readl(pci_ctrl->mem_base + DEVDRV_HOST_PHY_MACH_FLAG_OFFSET);
    if (*host_flag == BAR_EXCEP_VAL_32_BIT) {
        devdrv_err("*host_flag=0x%x, error!\n", *host_flag);
        return -EINVAL;
    }

    return 0;
}
EXPORT_SYMBOL(devdrv_get_host_phy_mach_flag);

/* set host flag */
void devdrv_set_host_phy_mach_flag(struct devdrv_pci_ctrl *pci_ctrl, u32 host_flag)
{
    writel(host_flag, (pci_ctrl->mem_base + DEVDRV_HOST_PHY_MACH_FLAG_OFFSET));
}

/*
    one device only surport a txatu item, repeat configuration will overwrite previous
    devid: host devid, size <= 128MB
    the input-argument-pid is useless and it's just for union.
*/
int devdrv_device_txatu_config(int pid, u32 devid, phys_addr_t host_phy_addr, u32 size)
{
    struct devdrv_ctrl *ctrl = devdrv_get_bottom_half_devctrl_by_id(devid);
    struct devdrv_pci_ctrl *pci_ctrl = NULL;
    int ret;

    if (ctrl == NULL) {
        devdrv_err("devid %d get ctrl failed!\n", devid);
        return -EINVAL;
    }

    pci_ctrl = (struct devdrv_pci_ctrl *)ctrl->priv;
    if (pci_ctrl->shr_para->chip_type == (int)BAR_EXCEP_VAL_32_BIT) {
        devdrv_err("devid %u, chip_type %d, error\n", devid, pci_ctrl->shr_para->chip_type);
        return -ENODEV;
    }
    if (pci_ctrl->shr_para->chip_type == HISI_MINI) {
        return 0;
    }

    if ((size == 0) || ((size % ATU_SIZE_ALIGN) != 0)) {
        devdrv_err("devid %d size 0x%x is illegal\n", devid, size);
        return -EINVAL;
    }

    ret = devdrv_set_tx_atu(pci_ctrl->io_base, pci_ctrl->dev_id, (u32)(-1), ATU_TYPE_TX_HOST,
        host_phy_addr, size, pci_ctrl->shr_para);
#ifndef DRV_UT

    if (ret) {
        devdrv_err("devid %d tx atu config failed, host_phy_addr, size 0x%x.\n", devid, size);
        return ret;
    }
#endif
    ret = devdrv_notify_dev_online(pci_ctrl->msg_dev, (u32)(-1), DEVDRV_DEV_ONLINE);

    if (ret) {
        devdrv_err("devid %d notify online failed, host_phy_addr, size 0x%x.\n", devid, size);
    }

    return ret;
}
EXPORT_SYMBOL(devdrv_device_txatu_config);

/* stub for svm */
int devdrv_devmem_addr_bar_to_dma(u32 devid, u32 dst_devid, phys_addr_t host_bar_addr, dma_addr_t *dma_addr)
{
    return -EINVAL;
}
EXPORT_SYMBOL(devdrv_devmem_addr_bar_to_dma);

/* stub for svm */
int devdrv_set_module_init_finish(int dev_id, int module)
{
    return 0;
}
EXPORT_SYMBOL(devdrv_set_module_init_finish);

/* This function called by host to get device index with host device id
 * on cloud,  an smp system has 4 chips, so the device index is 0 ~ 3
 * on mini, device index is 0
 */
int devdrv_get_device_index(u32 host_dev_id)
{
    int index = 0;
    return index;
}
EXPORT_SYMBOL(devdrv_get_device_index);

int devdrv_judge_pci_link_up(u32 devid)
{
    int ret = 0;
    unsigned short val = 0;
    struct devdrv_ctrl *ctrl = NULL;
    struct pci_dev *pdev = NULL;
    if (devid >= MAX_DEV_CNT) {
        devdrv_err("devid %u, invalid\n", devid);
        return -EINVAL;
    }
    ctrl = devdrv_get_top_half_devctrl_by_id(devid);
    if (ctrl == NULL) {
        devdrv_err("devid %u, get devctrl by id err\n", devid);
        return -ENODEV;
    }
    pdev = ctrl->pdev;
    if (pdev == NULL) {
        devdrv_err("devid %u, pdev NULL\n", devid);
        return -ENODEV;
    }
    ret = pci_read_config_word(pdev, PCI_DEVICE_ID, &val);
    if (ret) {
        devdrv_err("devid:%u, pci_read_config_word PCI_DEVICE_ID failed,ret=%d\n", devid, ret);
        return DEVDRV_PCI_LINK_DOWN;
    } else if (val != HISI_EP_DEVICE_ID) {
        return DEVDRV_PCI_LINK_DOWN;
    }
    return DEVDRV_PCI_LINK_UP;
}
EXPORT_SYMBOL(devdrv_judge_pci_link_up);

int devdrv_get_heartbeat_count(u32 devid, u64 *count)
{
    struct devdrv_pci_ctrl *pci_ctrl = NULL;
    struct devdrv_ctrl *ctrl = NULL;
    int ret;

    if (devid >= MAX_DEV_CNT) {
        devdrv_err("dev id is invalid\n");
        return -EINVAL;
    }
    if (count == NULL) {
        devdrv_err("heartbeat count is null\n");
        return -EINVAL;
    }
    ctrl = devdrv_get_top_half_devctrl_by_id(devid);
    if (ctrl == NULL) {
        devdrv_err("get devctrl by id err:%u\n", devid);
        return -ENODEV;
    }

    pci_ctrl = ctrl->priv;
    *count = pci_ctrl->shr_para->heartbeat_count;
    rmb();

    if (*count == BAR_EXCEP_VAL_64_BIT) {
        ret = devdrv_judge_pci_link_up(devid);
        if (ret == DEVDRV_PCI_LINK_DOWN) {
            devdrv_err("devid %u, pci link down\n", devid);
            return -ENODEV;
        }
    }

    return 0;
}
EXPORT_SYMBOL(devdrv_get_heartbeat_count);

int devdrv_get_runtime_runningplat(u32 devid, u64 *running_plat)
{
    struct devdrv_pci_ctrl *pci_ctrl = NULL;
    struct devdrv_ctrl *ctrl = NULL;

    if (devid >= MAX_DEV_CNT) {
        devdrv_err("dev id is invalid\n");
        return -EINVAL;
    }
    if (running_plat == NULL) {
        devdrv_err("dev(%u) invaild input, running_plat is null\n", devid);
        return -EINVAL;
    }
    ctrl = devdrv_get_top_half_devctrl_by_id(devid);
    if (ctrl == NULL) {
        devdrv_err("get devctrl by id err:dev%u\n", devid);
        return -ENODEV;
    }

    pci_ctrl = ctrl->priv;
    *running_plat = pci_ctrl->shr_para->runtime_runningplat;
    rmb();

    if (*running_plat == BAR_EXCEP_VAL_64_BIT) {
        devdrv_err("dev(%u), *running_plat 0x%llx, error\n", devid, *running_plat);
        return -ENODEV;
    }

    return 0;
}
EXPORT_SYMBOL(devdrv_get_runtime_runningplat);

int devdrv_set_runtime_runningplat(u32 devid, u64 running_plat)
{
    struct devdrv_pci_ctrl *pci_ctrl = NULL;
    struct devdrv_ctrl *ctrl = NULL;

    if (devid >= MAX_DEV_CNT) {
        devdrv_err("dev id is invalid\n");
        return -EINVAL;
    }

    ctrl = devdrv_get_top_half_devctrl_by_id(devid);
    if (ctrl == NULL) {
        devdrv_err("get devctrl by id err:dev%u\n", devid);
        return -ENODEV;
    }

    pci_ctrl = ctrl->priv;
    pci_ctrl->shr_para->runtime_runningplat = running_plat;
    wmb();

    return 0;
}
EXPORT_SYMBOL(devdrv_set_runtime_runningplat);
